package engine

import (
        "log"
    "spider/concurrent/fetcher"
    "spider/concurrent/parser"
    "spider/concurrent/scheduler"
)

type ConcurrentEngine struct{
    // 调度器接口
    Scheduler scheduler.Scheduler
    WorkerCount int
}

func (ConcurrentEngine *ConcurrentEngine) Run(seeds ...parser.Request) {
    // 创建输入输出channel
    in := make(chan parser.Request)
    out := make(chan parser.ParseResult)
    
    // 创建worker
    ConcurrentEngine.initWorker(in, out)
    
    // 设置调度器分发任务的channel
    ConcurrentEngine.Scheduler.SetMasterWorkerChan(in)
    
    // 提交request给带调度器
    for _, request := range seeds {
        ConcurrentEngine.Scheduler.Submit(request)
    }
    
    counter := 0
    // 处理worker执行结果
    for  {
        // 注意此处每获取一次结果(result := <- out)，需要将结果打印出来，并且将新的request分发出去
        // 当无空闲worker时，会无法将新的request分发出去,会产生worker之间循环等待
        // 因此可以使用goroutine分发任务，防止循环等待
        // 具体看慕课谷歌go视频25-2 15分钟后半段
        result := <- out
        for _, item := range result.ReponseSlice {
            log.Printf("Got item #%d: %v\n",counter,item)
            counter++
        }
    
        for _, request := range result.Requests {
            ConcurrentEngine.Scheduler.Submit(request)
        }
    }
}

// 初始化worker
func (ConcurrentEngine *ConcurrentEngine) initWorker(in chan parser.Request, out chan parser.ParseResult) {
    for i := 0; i < ConcurrentEngine.WorkerCount; i++ {
        createWorker(in, out)
    }
}

// 创建worker
func createWorker(in chan parser.Request,out chan parser.ParseResult)  {
    go func() {
        for {
            request := <- in
            
            result, err := worker(request)
            if err != nil {
                continue
            }
            
            out <- result
        }
    }()
}

func worker(request parser.Request) (parser.ParseResult, error) {
    
    //  获取内容
    log.Printf("Fetching %s",request.Url)
    content, err := fetcher.Fetch(request.Url)
    if err != nil {
        log.Printf("Fetcher: error fetching url %s : %v",request.Url,err)
        return parser.ParseResult{},nil
    }
    
    // 解析内容
    parseResult := request.Parser.Analysis(content)
    
    return parseResult,nil
}